# -*- coding: utf-8 -*-
import scrapy
import os

from dra100_scrapy.items import ScrapyItem
from scrapy.http import Request
from scrapy.utils.project import get_project_settings

class YigoonetSpider(scrapy.Spider):
    #医谷
    name = "yigoonet"
    site_name = "医谷"
    allowed_domains = ["yigoonet.com"]
    start_urls = ['http://www.yigoonet.com/news/']

    # 列表
    def parse(self, response):
        settings = get_project_settings()
        article_list = response.xpath('//div[@class="articleList"]/article')
        i = 0
        for article in article_list:
            img_url = article.xpath('p[@class="pic"]/a/img/@src').extract()[0]
            link = article.xpath('p[@class="pic"]/a/@href').extract()[0]
            summary = article.xpath('p[@class="dec"]/text()').extract()[0]

            #记录第一行url避免重复抓取
            if i == 0: 
                log_file_name = settings['SPIDER_LOG_STORE'] + self.name
                f = open(log_file_name, 'r+')
                log_link = f.read()
                if log_link == link:
                    break
                else:
                    f.seek(0)
                    f.truncate()
                    f.write(link)
                f.close()

            yield Request(link, callback=self.parse_detail, meta={'img_url': img_url, 'link': link, 'summary': summary})
            i += 1

    # 详情
    def parse_detail(self, response):
        item = ScrapyItem()
        img_url = response.meta['img_url']
        item['img_url'] = ''
        item['summary'] = response.meta['summary']
        item['source_url'] = response.meta['link']
        item['source'] = self.site_name
        item['title'] = response.xpath('//section[@class="showList"]/article/h1/text()').extract()[0].strip()

        p_list = response.xpath('//div[@id="dec" and @class="dec"]/p').extract()
        item['content'] = "".join(p_list)

        img_url_list = response.xpath('//div[@id="dec" and @class="dec"]/p//img/@src').extract()
        if (len(img_url) != 0):
            img_url_list.insert(0, response.meta['img_url'])
        
        item['image_urls'] = img_url_list
        
        return item